/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2011-3-25
 */
package code.extract.training;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

public class PageLinkList {
	private HashSet<String> keywordSet = new HashSet<String>();
	private HashMap<String, ArrayList<String> > keywordLinkList = new HashMap<String, ArrayList<String>>();
	private HashSet<String> crawledKeywordSet = new HashSet<String>();
	private HashSet<String> crawledLinkSet = new HashSet<String>();
//	private String linklistFilename;
	private String crawledKeywordFilename;
	private String crawledLinklistFilename;
	private String encode;
	private BufferedWriter crawledKeywordBW;
	private BufferedWriter crawledLinkBW;
	public PageLinkList(String linklistFilename, String crawledKeywordFilename, 
			String crawledLinklistFilename, String encode){
		initLinkList(linklistFilename, encode);
		initCrawledLinkSet(crawledLinklistFilename, encode);
		initCrawledKeywordSet(crawledKeywordFilename, encode);
//		this.linklistFilename = linklistFilename;
		this.crawledKeywordFilename = crawledKeywordFilename;
		this.crawledLinklistFilename = crawledLinklistFilename;
		this.encode = encode;
		initBW(this.crawledKeywordFilename, this.encode, this.crawledKeywordBW);
		initBW(this.crawledLinklistFilename, this.encode, this.crawledLinkBW);
	}
	public int initBW(String filename, String encode, BufferedWriter bw){
		int result = 0;
		try{
			File file = new File(filename);
			FileOutputStream fos = new FileOutputStream(file);
			OutputStreamWriter osw = null;
			if (encode == null){
				osw = new OutputStreamWriter(fos);
			} else{
				osw = new OutputStreamWriter(fos, encode);
			}
			bw = new BufferedWriter(osw);
		}catch (Exception e) {
			result = 1;
		}
		return result;
	}
	public void closeBW(BufferedWriter bw){
		try{
			bw.close();
		}catch (Exception e) {
		}
	}
	public void close(){
		closeBW(crawledKeywordBW);
		closeBW(crawledLinkBW);
	}
	
	public int initCrawledKeywordSet(String filename, String encode){
		int result = 0;
		try{
			File file = new File(filename);
			FileInputStream fis = new FileInputStream(file);
			InputStreamReader isr = null;
			if (encode == null){
				isr = new InputStreamReader(fis);
			} else{
				isr = new InputStreamReader(fis, encode);
			}
			BufferedReader br = new BufferedReader(isr);
			crawledKeywordSet.clear();
			String line = br.readLine();
			while (line != null){
				crawledKeywordSet.add(line.trim());
				line = br.readLine();
			}
			br.close();
			isr.close();
			fis.close();
		}catch (Exception e) {
			result = 1;
		}
		return result;
	}
	public int initCrawledLinkSet(String filename, String encode){
		int result = 0;
		try{
			File file = new File(filename);
			FileInputStream fis = new FileInputStream(file);
			InputStreamReader isr = null;
			if (encode == null){
				isr = new InputStreamReader(fis);
			} else{
				isr = new InputStreamReader(fis, encode);
			}
			BufferedReader br = new BufferedReader(isr);
			crawledLinkSet.clear();
			String line = br.readLine();
			while (line != null){
				crawledLinkSet.add(line.trim());
				line = br.readLine();
			}
			br.close();
			isr.close();
			fis.close();
		}catch (Exception e) {
			result = 1;
		}
		return result;
	}
	public int initLinkList(String filename, String encode){
		int result = 0;
		try{
			File file = new File(filename);
			FileInputStream fis = new FileInputStream(file);
			InputStreamReader isr = null;
			if (encode == null){
				isr = new InputStreamReader(fis);
			} else{
				isr = new InputStreamReader(fis, encode);
			}
			BufferedReader br = new BufferedReader(isr);
			keywordLinkList.clear();
			keywordSet.clear();
			String line = br.readLine();
			ArrayList<String> thisLinkList = new ArrayList<String>();
			String thisKeyword = "";
			while (line != null){
				line = line.trim();
				if (line.length() > 0)
				{
					if (line.startsWith(TrainLinkCrawler.KEYWORD_START_SYMBOL)){
						if (thisLinkList.size() > 0 && thisKeyword.length() > 0){
							keywordLinkList.put(thisKeyword, thisLinkList);
						}
						thisKeyword = line.replace(TrainLinkCrawler.KEYWORD_START_SYMBOL, "");
						thisLinkList = new ArrayList<String>();
						keywordSet.add(thisKeyword);
					}else{
						thisLinkList.add(line.trim());
					}
				}
				line = br.readLine();
			}
			if (thisLinkList.size() > 0 && thisKeyword.length() > 0){
				keywordLinkList.put(thisKeyword, thisLinkList);
			}
			br.close();
			isr.close();
			fis.close();
		}catch (Exception e) {
			result = 1;
		}
		return result;
	}
	public synchronized void addCrawledKeyword(String keyword){
		if (crawledKeywordSet.contains(keyword)){
			return;
		}else{
			crawledKeywordSet.add(keyword);
			try{
				crawledKeywordBW.write(keyword);
				crawledKeywordBW.write("\n");
				crawledKeywordBW.flush();
			}catch (Exception e) {
				e.printStackTrace();
			}
		}
	}
	public synchronized void addCrawledLink(String link){
		if (crawledLinkSet.contains(link)){
			return;
		}else{
			crawledLinkSet.add(link);
			try{
				crawledLinkBW.write(link);
				crawledLinkBW.write("\n");
				crawledLinkBW.flush();
			}catch (Exception e) {
				e.printStackTrace();
			}
		}
	}
	public synchronized void addCrawledLinklist(ArrayList<String> linklist){
		StringBuffer sb = new StringBuffer();
		for (int i=0; i<linklist.size(); ++i){
			String thisLink = linklist.get(i);
			if (crawledLinkSet.contains(thisLink)){
				continue;
			}else{
				crawledLinkSet.add(thisLink);
				sb.append(thisLink);
				sb.append("\n");
			}
		}
		try{
			crawledLinkBW.write(sb.toString());
			crawledLinkBW.flush();
		}catch (Exception e) {
			e.printStackTrace();
		}
	}
	public HashSet<String> getKeywordSet() {
		return keywordSet;
	}
	public HashMap<String, ArrayList<String>> getKeywordLinkList() {
		return keywordLinkList;
	}
	public HashSet<String> getCrawledKeywordSet() {
		return crawledKeywordSet;
	}
	public HashSet<String> getCrawledLinkSet() {
		return crawledLinkSet;
	}
	
}
